"""
This script looks for data and seeks to pickle it using cPickle and
place the pickles in a 'mirrored' version of the database file structure.
"""
import os
import cPickle

#this_db_path = "/experiment_data/stored_data"
this_db_path = "/experiment_data/test_database"

to_be_pickled = []
already_pickled = []
# find all the directories with 'session' in their names.
files = os.listdir(this_db_path)
last_data_path = ''
for file in files:
    if 'session' in file:
        #print "found file folder called: %s" % file
        # find folders in this directory with 'experiment' in their names
        efiles = os.listdir(os.path.join(this_db_path,file))
        for efile in efiles:
            if 'experiment' in efile:
                #print "    found file folder called: %s" % efile
                data_path = os.path.join(this_db_path, file)
                data_path = os.path.join(data_path, efile)
                # check to see if any data has been pickled into this pickle_path
                dfiles = os.listdir(data_path)
                for dfile in dfiles:
                    if 'data' in dfile and '.cPickle' == dfile[-8:]:
                        if 'data_pickle_tested' in dfiles:
                            already_pickled.append({'data_path':data_path, 'data_filename':dfile})
                    elif 'pickle' not in dfile and 'Pickle' not in dfile:
                        if last_data_path == data_path:
                            #FIXME find out what happened for these files, and see if there are any like it in Dan's DB
                            print data_path
                        to_be_pickled.append({'data_path':data_path, 'data_filename':dfile})
                        last_data_path = data_path

print "Found %d data files to be pickled.  While %d data files have already been pickled." % (len(to_be_pickled), len(already_pickled))

# go about actually pickling the data files.
num_jobs = len(to_be_pickled)
for i in xrange(num_jobs):
    this_job = to_be_pickled.pop()
    #print "  in folder %s" % this_job['data_path'][-30:]                    
    #print this_job['data_filename'][:10]
    already_pickled.append(this_job)
    
#print "Found %d data files to be pickled.  While %d data files have already been pickled." % (len(to_be_pickled), len(already_pickled))




